{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.00809361, -0.01217612, -0.00668608, -0.03422887], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAT1UlEQVR4nO3db2xT970G8Md2YkP+HKchi72MeKC1Ks2FsC1AcsaLVcMj66JtrLlX24S6rOJSlTmokAlpuWpBoEpB9ErdutHwYhtUkzqmTGJrc2lRlLThTjWkpMsUAuR2WqtkJbb5Ux+HQJzE/t4X3Jw7l0DjJPhnx89HOohzfl/b33OInxyfn40tIiIgIlLAqroBIspeDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlJGWQAdPnwYK1aswJIlS1BdXY2enh5VrRCRIkoC6Pe//z2ampqwb98+vPfee1i7di1qa2sRCoVUtENEilhUfBi1uroa69evxy9/+UsAQDweR3l5OXbu3Imf/vSnqW6HiBTJSfUDTkxMoLe3F83NzeY2q9UKr9cLv98/422i0Sii0ai5Ho/Hcf36dSxbtgwWi+W+90xEyRERjI6OoqysDFbr3V9opTyArl69ilgsBpfLlbDd5XLh0qVLM96mpaUF+/fvT0V7RLSAhoeHsXz58ruOpzyA5qK5uRlNTU3mumEY8Hg8GB4ehqZpCjsjoplEIhGUl5ejsLDwnnUpD6CSkhLYbDYEg8GE7cFgEG63e8bbOBwOOByOO7ZrmsYAIkpjn3aJJOWzYHa7HVVVVejs7DS3xeNxdHZ2Qtf1VLdDRAopeQnW1NSEhoYGrFu3Dhs2bMDPfvYzjI2N4cknn1TRDhEpoiSAvve97+HKlSvYu3cvAoEAvvjFL+LNN9+848I0ES1uSt4HNF+RSAROpxOGYfAaEFEamu1zlJ8FIyJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTJJB9Dp06fxrW99C2VlZbBYLPjjH/+YMC4i2Lt3Lz772c9i6dKl8Hq9eP/99xNqrl+/jq1bt0LTNBQVFWHbtm24cePGvHaEiDJP0gE0NjaGtWvX4vDhwzOOHzp0CC+99BKOHDmCs2fPIj8/H7W1tRgfHzdrtm7dioGBAXR0dKC9vR2nT5/GU089Nfe9IKLMJPMAQE6cOGGux+Nxcbvd8sILL5jbwuGwOBwO+d3vficiIhcuXBAA8u6775o1b7zxhlgsFvnoo49m9biGYQgAMQxjPu0T0X0y2+fogl4D+uCDDxAIBOD1es1tTqcT1dXV8Pv9AAC/34+ioiKsW7fOrPF6vbBarTh79uyM9xuNRhGJRBIWIsp8CxpAgUAAAOByuRK2u1wucywQCKC0tDRhPCcnB8XFxWbNJ7W0tMDpdJpLeXn5QrZNRIpkxCxYc3MzDMMwl+HhYdUtEdECWNAAcrvdAIBgMJiwPRgMmmNutxuhUChhfGpqCtevXzdrPsnhcEDTtISFiDLfggbQypUr4Xa70dnZaW6LRCI4e/YsdF0HAOi6jnA4jN7eXrOmq6sL8Xgc1dXVC9kOEaW5nGRvcOPGDfztb38z1z/44AP09fWhuLgYHo8Hu3btwvPPP4+HHnoIK1euxHPPPYeysjJs2bIFAPDII4/gG9/4BrZv344jR45gcnISjY2N+P73v4+ysrIF2zEiygDJTq+99dZbAuCOpaGhQURuT8U/99xz4nK5xOFwyKZNm2RwcDDhPq5duyY/+MEPpKCgQDRNkyeffFJGR0cXfIqPiNSY7XPUIiKiMP/mJBKJwOl0wjAMXg8iSkOzfY5mxCwYES1ODCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhImaS/lodooUzeGkXkHxfMdYvVBufnK2HLsSvsilKJAUTKjIcD+HvXr811m30p/uXf9sFWUKywK0olvgSj9JJ53xJF88AAovQh099zSdmCAUTphWdAWYUBRGlDAGTgF/XSPDCAKM0wgLIJA4jSCs+AsktSAdTS0oL169ejsLAQpaWl2LJlCwYHBxNqxsfH4fP5sGzZMhQUFKC+vh7BYDChZmhoCHV1dcjLy0NpaSn27NmDqamp+e8NZTjhNaAsk1QAdXd3w+fz4cyZM+jo6MDk5CQ2b96MsbExs2b37t14/fXX0dbWhu7ubly+fBmPP/64OR6LxVBXV4eJiQm88847eOWVV3Ds2DHs3bt34faKMhgDKKvIPIRCIQEg3d3dIiISDoclNzdX2trazJqLFy8KAPH7/SIicvLkSbFarRIIBMya1tZW0TRNotHorB7XMAwBIIZhzKd9Uixy+X+k58h2czn360YZu/YP1W3RApjtc3Re14AMwwAAFBfffudqb28vJicn4fV6zZpVq1bB4/HA7/cDAPx+P9asWQOXy2XW1NbWIhKJYGBgYMbHiUajiEQiCQstQrenwVR3QSk05wCKx+PYtWsXNm7ciNWrVwMAAoEA7HY7ioqKEmpdLhcCgYBZ88/hMz0+PTaTlpYWOJ1OcykvL59r25TuGEBZZc4B5PP5cP78eRw/fnwh+5lRc3MzDMMwl+Hh4fv+mKSG8BpQVpnTh1EbGxvR3t6O06dPY/ny5eZ2t9uNiYkJhMPhhLOgYDAIt9tt1vT09CTc3/Qs2XTNJzkcDjgcjrm0ShmFs2DZJqkzIBFBY2MjTpw4ga6uLqxcuTJhvKqqCrm5uejs7DS3DQ4OYmhoCLquAwB0XUd/fz9CoZBZ09HRAU3TUFFRMZ99oUWBAZRNkjoD8vl8ePXVV/GnP/0JhYWF5jUbp9OJpUuXwul0Ytu2bWhqakJxcTE0TcPOnTuh6zpqamoAAJs3b0ZFRQWeeOIJHDp0CIFAAM8++yx8Ph/PcohvRMwySQVQa2srAODRRx9N2H706FH86Ec/AgC8+OKLsFqtqK+vRzQaRW1tLV5++WWz1mazob29HTt27ICu68jPz0dDQwMOHDgwvz2hxYEBlFUskoG/ciKRCJxOJwzDgKZpqtuhORodeR+XXnvBXLfm2PFw3W4UuL+gsCtaCLN9jvKzYJQ2RISzYFmGAUTpJfNOyGkeGECUVjLwigDNAwOI0gwDKJswgCi98AwoqzCASB2L5fZiEkg8pqwdSj0GECmTu1RDjqPAXJfYFKKRKwo7olRjAJEyFqsNFmvijyDPgLILA4gUsnx6CS1qDCBSx2L+QVmKAUTKWGBh/mQ5BhCpY7GACZTdGECklMXCAMpmDCBSh+GT9RhApIwFfAmW7RhApA7zJ+sxgEghJlC2YwCROhYL4yfLMYBImdvvQ2QEZTMGECnE8Ml2DCBS547/joOyDQOIFGMAZTMGEClj4UXorMcAIoX4EizbJfXNqETJiMfjGB0dves3XcSnoojF4gnbbt26hXA4PGO9zWZDQUEBPz+2iDCA6L75+OOPsWnTJly9enXGcUeOFYf+fSPKSwrNbS0HD+K/ej6csb6yshKvvfYacnL4Y7tY8F+S7pt4PI6RkRGEQqEZxx25NoRuliAUeRQ5lkk8mNcLIxzGRx99NGO92+2+n+2SAkldA2ptbUVlZSU0TYOmadB1HW+88YY5Pj4+Dp/Ph2XLlqGgoAD19fUIBoMJ9zE0NIS6ujrk5eWhtLQUe/bswdTU1MLsDWUUrWg5/mH9Nq5Mfh4jE1/AX0c3YVIcqtuiFEoqgJYvX46DBw+it7cX586dw9e+9jV85zvfwcDAAABg9+7deP3119HW1obu7m5cvnwZjz/+uHn7WCyGuro6TExM4J133sErr7yCY8eOYe/evQu7V5QRluYVwb7E+X9rFtyMFyImPCnPKjJPDzzwgPzqV7+ScDgsubm50tbWZo5dvHhRAIjf7xcRkZMnT4rVapVAIGDWtLa2iqZpEo1GZ/2YhmEIADEMY77t030UCoWktLRUcPvrTu9Y8vOL5D/2/Fb2P39WDjx/Vn79wm/lXx+tvGt9VVWVTE5Oqt4tmoXZPkfn/OsmFouhra0NY2Nj0HUdvb29mJychNfrNWtWrVoFj8cDv9+Pmpoa+P1+rFmzBi6Xy6ypra3Fjh07MDAwgC996UtJ9XDp0iUUFBR8eiEp8fHHH9/z5fXNm2H0/vd/YsLxCGyWSbjtH2Lg75fvWj8+Po6LFy/CZrPdj3ZpAd24cWNWdUkHUH9/P3Rdx/j4OAoKCnDixAlUVFSgr68PdrsdRUVFCfUulwuBQAAAEAgEEsJnenx67G6i0Sii0ai5HolEAACGYfD6URqLRCJ3nYIHbn8L86kzfwXw11ndXywWg2EYsFr59rV0NzY2Nqu6pAPo4YcfRl9fHwzDwB/+8Ac0NDSgu7s76QaT0dLSgv3799+xvbq6Gpqm3dfHprm7cuUKcnNzF+z+8vPzUVNTw2n4DDB9kvBpkv5VYrfb8eCDD6KqqgotLS1Yu3Ytfv7zn8PtdmNiYuKON5EFg0Fz+tTtdt8xKza9fq8p1ubmZhiGYS7Dw8PJtk1EaWje57LxeBzRaBRVVVXIzc1FZ2enOTY4OIihoSHoug4A0HUd/f39Ce8L6ejogKZpqKiouOtjOBwOc+p/eiGizJfUuWxzczMee+wxeDwejI6O4tVXX8Xbb7+NU6dOwel0Ytu2bWhqakJxcTE0TcPOnTuh6zpqamoAAJs3b0ZFRQWeeOIJHDp0CIFAAM8++yx8Ph8cDr7/gyjbJBVAoVAIP/zhDzEyMgKn04nKykqcOnUKX//61wEAL774IqxWK+rr6xGNRlFbW4uXX37ZvL3NZkN7ezt27NgBXdeRn5+PhoYGHDhwYGH3itKCxWJBYWEhxsfHF+T+OOO5+FjkXtMUaSoSicDpdMIwDL4cS2OxWAyBQADxePzTi2fBbrejtLSUH0bNALN9jnI6ge4bm82Gz33uc6rboDTGN1QQkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZXJUNzAXIgIAiEQiijshoplMPzenn6t3k5EBdO3aNQBAeXm54k6I6F5GR0fhdDrvOp6RAVRcXAwAGBoauufOUaJIJILy8nIMDw9D0zTV7WQEHrO5ERGMjo6irKzsnnUZGUBW6+1LV06nkz8Uc6BpGo9bknjMkjebkwNehCYiZRhARKRMRgaQw+HAvn374HA4VLeSUXjcksdjdn9Z5NPmyYiI7pOMPAMiosWBAUREyjCAiEgZBhARKZORAXT48GGsWLECS5YsQXV1NXp6elS3pExLSwvWr1+PwsJClJaWYsuWLRgcHEyoGR8fh8/nw7Jly1BQUID6+noEg8GEmqGhIdTV1SEvLw+lpaXYs2cPpqamUrkryhw8eBAWiwW7du0yt/GYpYhkmOPHj4vdbpff/OY3MjAwINu3b5eioiIJBoOqW1OitrZWjh49KufPn5e+vj755je/KR6PR27cuGHWPP3001JeXi6dnZ1y7tw5qampka985Svm+NTUlKxevVq8Xq/85S9/kZMnT0pJSYk0Nzer2KWU6unpkRUrVkhlZaU888wz5nYes9TIuADasGGD+Hw+cz0Wi0lZWZm0tLQo7Cp9hEIhASDd3d0iIhIOhyU3N1fa2trMmosXLwoA8fv9IiJy8uRJsVqtEggEzJrW1lbRNE2i0WhqdyCFRkdH5aGHHpKOjg756le/agYQj1nqZNRLsImJCfT29sLr9ZrbrFYrvF4v/H6/ws7Sh2EYAP7/A7u9vb2YnJxMOGarVq2Cx+Mxj5nf78eaNWvgcrnMmtraWkQiEQwMDKSw+9Ty+Xyoq6tLODYAj1kqZdSHUa9evYpYLJbwjw4ALpcLly5dUtRV+ojH49i1axc2btyI1atXAwACgQDsdjuKiooSal0uFwKBgFkz0zGdHluMjh8/jvfeew/vvvvuHWM8ZqmTUQFE9+bz+XD+/Hn8+c9/Vt1KWhseHsYzzzyDjo4OLFmyRHU7WS2jXoKVlJTAZrPdMRsRDAbhdrsVdZUeGhsb0d7ejrfeegvLly83t7vdbkxMTCAcDifU//Mxc7vdMx7T6bHFpre3F6FQCF/+8peRk5ODnJwcdHd346WXXkJOTg5cLhePWYpkVADZ7XZUVVWhs7PT3BaPx9HZ2Qld1xV2po6IoLGxESdOnEBXVxdWrlyZMF5VVYXc3NyEYzY4OIihoSHzmOm6jv7+foRCIbOmo6MDmqahoqIiNTuSQps2bUJ/fz/6+vrMZd26ddi6dav5dx6zFFF9FTxZx48fF4fDIceOHZMLFy7IU089JUVFRQmzEdlkx44d4nQ65e2335aRkRFzuXnzplnz9NNPi8fjka6uLjl37pzoui66rpvj01PKmzdvlr6+PnnzzTflM5/5TFZNKf/zLJgIj1mqZFwAiYj84he/EI/HI3a7XTZs2CBnzpxR3ZIyAGZcjh49atbcunVLfvzjH8sDDzwgeXl58t3vfldGRkYS7ufDDz+Uxx57TJYuXSolJSXyk5/8RCYnJ1O8N+p8MoB4zFKD/x0HESmTUdeAiGhxYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETK/C/Sxok4ca3I/gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38], [4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38], (4,), float32)\n",
      "env.action_space= Discrete(2)\n",
      "state= [ 0.03104177 -0.00820064  0.01764811 -0.02930056]\n",
      "action= 0\n",
      "next_state= [ 0.03087776 -0.20357117  0.0170621   0.26889786]\n",
      "reward= 1.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.a2c.a2c.A2C at 0x7fbb8b9c5e80>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import A2C\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = A2C(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=5,  #运行N步后执行更新,batch_size=n_steps*环境数量\n",
    "    gamma=0.9,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(22.05, 9.107551811546283)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(200_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/2.A2C.CartPole')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(195.55, 12.355059692288012)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = A2C.load('save/2.A2C.CartPole')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAVD0lEQVR4nO3dfWxT570H8K/t2M7rcZqE2MuIB1ejo7m8rQESt39st83IaFaNNn9sE7fLEKIX5qDSTEiN1ILabUrHpHbrSuHeaYXuTh1TJrGpEW2XG0rQhiGQLlJISm670ZsMsA2EHDshfn/uHywHTBKIE5MnJ/1+pCPh53ls/84Bf7Gf4/PYIIQQICKSwCi7ACL67GIAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNNICaO/evVi0aBEyMzNRUVGBjo4OWaUQkSRSAuh3v/sdGhoasHv3bnz44YdYuXIlqqur4ff7ZZRDRJIYZFyMWlFRgTVr1uD1118HACQSCZSWlmL79u147rnnZrscIpIkY7afMBKJoLOzE42NjVqb0WhEVVUVPB7PhPcJh8MIh8Pa7UQigcHBQRQWFsJgMNzzmokoNUIIBINBlJSUwGic/IPWrAfQlStXEI/HYbfbk9rtdjvOnTs34X2amprw4osvzkZ5RJRGAwMDWLhw4aT9sx5A09HY2IiGhgbttqqqcDqdGBgYgKIoEisjookEAgGUlpYiLy/vjuNmPYCKiopgMpng8/mS2n0+HxwOx4T3sVqtsFqt49oVRWEAEc1hd5simfWzYBaLBeXl5Whra9PaEokE2tra4HK5ZrscIpJIykewhoYG1NXVYfXq1Vi7di1+9rOfYWRkBJs2bZJRDhFJIiWAvvWtb+Hy5cvYtWsXvF4vVq1ahffee2/cxDQRzW9Svgc0U4FAADabDaqqcg6IaA6a6muU14IRkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhImpQD6Pjx43j88cdRUlICg8GAP/zhD0n9Qgjs2rULn/vc55CVlYWqqip8/PHHSWMGBwexceNGKIqC/Px8bN68GcPDwzPaESLSn5QDaGRkBCtXrsTevXsn7N+zZw9ee+017N+/H6dOnUJOTg6qq6sRCoW0MRs3bkRPTw9aW1vR0tKC48eP4+mnn57+XhCRPokZACAOHz6s3U4kEsLhcIif/vSnWtvQ0JCwWq3it7/9rRBCiN7eXgFAnD59Whvz7rvvCoPBIC5cuDCl51VVVQAQqqrOpHwiukem+hpN6xzQ+fPn4fV6UVVVpbXZbDZUVFTA4/EAADweD/Lz87F69WptTFVVFYxGI06dOjXh44bDYQQCgaSNiPQvrQHk9XoBAHa7PandbrdrfV6vF8XFxUn9GRkZKCgo0MbcrqmpCTabTdtKS0vTWTYRSaKLs2CNjY1QVVXbBgYGZJdERGmQ1gByOBwAAJ/Pl9Tu8/m0PofDAb/fn9Qfi8UwODiojbmd1WqFoihJGxHpX1oDaPHixXA4HGhra9PaAoEATp06BZfLBQBwuVwYGhpCZ2enNubo0aNIJBKoqKhIZzlENMdlpHqH4eFhfPLJJ9rt8+fPo6urCwUFBXA6ndixYwd+9KMfYcmSJVi8eDFeeOEFlJSUYMOGDQCABx54AF//+texZcsW7N+/H9FoFPX19fj2t7+NkpKStO0YEelAqqfXPvjgAwFg3FZXVyeEuHEq/oUXXhB2u11YrVbx6KOPir6+vqTHuHr1qvjOd74jcnNzhaIoYtOmTSIYDKb9FB8RyTHV16hBCCEk5t+0BAIB2Gw2qKrK+SCiOWiqr1FdnAUjovmJAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJk/LP8tDsEkIgFhq+uYVHkLPgC7Dk3Ce7NKIZYwDNcfHIdfS1vILI8DWIRAyJeAz/8m+bULiEP+JI+scAmuuEQHQ0iHjkutYUGb4msSCi9OEc0BxnMJlhzStMahu58n+SqiFKLwbQHGc0ZcCqLEhqE4kEdPh7kkTjMIDmOoMRJnNmUlMiFoZIxCUVRJQ+DKA5zmAwAAZDUltsNIhELCKpIqL0YQDpQFZBSdLtUOAy4pFRSdUQpU9KAdTU1IQ1a9YgLy8PxcXF2LBhA/r6+pLGhEIhuN1uFBYWIjc3F7W1tfD5fElj+vv7UVNTg+zsbBQXF2Pnzp2IxWIz35t5ypq3AIDhruOI9CalAGpvb4fb7cbJkyfR2tqKaDSKdevWYWRkRBvz7LPP4p133kFzczPa29tx8eJFPPnkk1p/PB5HTU0NIpEITpw4gbfeegsHDx7Erl270rdX80yGNTs5f0QC8WhIWj1EaSNmwO/3CwCivb1dCCHE0NCQMJvNorm5WRvz0UcfCQDC4/EIIYQ4cuSIMBqNwuv1amP27dsnFEUR4XB4Ss+rqqoAIFRVnUn5uhH0/V10/Od/iI79W0TH/i3i9H9tFdfOd8kui2hSU32NzmgOSFVVAEBBQQEAoLOzE9FoFFVVVdqYpUuXwul0wuPxAAA8Hg+WL18Ou92ujamurkYgEEBPT8+EzxMOhxEIBJK2zxJztg3mLEW7LRJxjKpeiRURpce0AyiRSGDHjh14+OGHsWzZMgCA1+uFxWJBfn5+0li73Q6v16uNuTV8xvrH+ibS1NQEm82mbaWlpdMtW5cyLNkwWbKSGwX4XSDSvWkHkNvtxtmzZ3Ho0KF01jOhxsZGqKqqbQMDA/f8OecSY4YZxgxzUhvngGg+mNa1YPX19WhpacHx48excOFCrd3hcCASiWBoaCjpXZDP54PD4dDGdHR0JD3e2FmysTG3s1qtsFqt0yl13oqMXAOEGPcdISI9SekdkBAC9fX1OHz4MI4ePYrFixcn9ZeXl8NsNqOtrU1r6+vrQ39/P1wuFwDA5XKhu7sbfr9fG9Pa2gpFUVBWVjaTfZm/DAZk2pI/to5eGYAQCUkFEaVHSu+A3G433n77bfzxj39EXl6eNmdjs9mQlZUFm82GzZs3o6GhAQUFBVAUBdu3b4fL5UJlZSUAYN26dSgrK8NTTz2FPXv2wOv14vnnn4fb7ea7nEkZkJmf/O6Q4UPzQUoBtG/fPgDAV7/61aT2AwcO4Hvf+x4A4NVXX4XRaERtbS3C4TCqq6vxxhtvaGNNJhNaWlqwbds2uFwu5OTkoK6uDi+99NLM9mSey7BmJ91OxGNIxKMwmriiCumXQejwVEogEIDNZoOqqlAU5e53mAd8Z4+i/y83J/wzshSUPfEcrHlFEqsimthUX6O8FkwnMm3FMNzybic2GuTCZKR7DCCdsOQUwGC89eOW+OdGpF8MIJ0wWbJuLM1xC14RT3rHANKLCdYFCqn+SQYT6QMDSCdM5kxYleKkttHBC5KqIUoPBpBOGExmmLPyxrXr8CQmkYYBpBMG4/i1oePRMMAvJJKOMYB04vYJaACIjgaRiHNxetIvBpCOZOYnXw8WGvIiEQtLqoZo5hhAOpJ1X/Li9BCJG1fEE+kUA0hHTJbk68FEgmtDk74xgPTktnkgkYghel2VVAzRzDGAdMSSkw/TLVfFJ2IRhIa4NjTpFwNIRzKy8mAyJ68NLYTgd4FItxhAOmLKsIxb/yce5fVgpF8MIF0Zfz1YWL0sqRaimWMA6YjBaERWweeT2q4P/gNcloP0igGkJwYjLDn3ya6CKG0YQDpz+9rQIhaFiMckVUM0MwwgHblxPdj4RcniEX4ZkfSJAaQzVqUIMNz8a4uMDCE6GpBYEdH0MYB0JjPfDoPRpN0WIsHfCCPdYgDpDNeGpvmEAaQ7t60LJAQiw4NySiGaIf6s5hwUCoUQCk08sRwLRWDMzENi+Oo/WwSuXfwEGcUPTPp42dnZsFgs96BSoplhAM1Bv/zlL/GTn/xkwr4MowG7N67Bv37h5i+i/vevf403/71x0sfbv38/vvGNb6S9TqKZYgDNQcFgEBcuTP6LF4PBVfj4ejmuxRywWz5FLPq/uHjxwqRrk42Oco6I5qaU5oD27duHFStWQFEUKIoCl8uFd999V+sPhUJwu90oLCxEbm4uamtr4fP5kh6jv78fNTU1yM7ORnFxMXbu3IlYjF+kS8XHI+X42+iDGIx+HudGKhHLXg2zyXT3OxLNMSkF0MKFC/Hyyy+js7MTZ86cwSOPPIJvfvOb6OnpAQA8++yzeOedd9Dc3Iz29nZcvHgRTz75pHb/eDyOmpoaRCIRnDhxAm+99RYOHjyIXbt2pXev5rnheD7GJqMFTFiw4AuwmBlApD8pBdDjjz+Oxx57DEuWLMH999+PH//4x8jNzcXJkyehqip+9atf4ZVXXsEjjzyC8vJyHDhwACdOnMDJkycBAH/605/Q29uL3/zmN1i1ahXWr1+PH/7wh9i7dy8ikcg92cH56PI//gITwhBCwJgIwmb4G9cEIl2a9hxQPB5Hc3MzRkZG4HK50NnZiWg0iqqqKm3M0qVL4XQ64fF4UFlZCY/Hg+XLl8Nuv/nrDtXV1di2bRt6enrw5S9/OaUazp07h9zc3Onuwpx1+fKdl9g41/s/MMZVBBMLoF7uxbXBv+N6ODrp+AsXLqC3tzfdZRJNanh4eErjUg6g7u5uuFwuhEIh5Obm4vDhwygrK0NXVxcsFgvy8/OTxtvtdni9N5YN9Xq9SeEz1j/WN5lwOIxw+ObPzwQCNy49UFV1Xs4f3W3SuN83hH5f65Qfb2RkBENDQzOsimjqRkZGpjQu5QD60pe+hK6uLqiqit///veoq6tDe3t7ygWmoqmpCS+++OK49oqKCiiKck+fW4Zjx46l9fHuv/9+PPTQQ2l9TKI7GXuTcDcpfxPaYrHgi1/8IsrLy9HU1ISVK1fi5z//ORwOByKRyLj/aX0+HxwOBwDA4XCMOys2dntszEQaGxuhqqq2DQwMpFo2Ec1BM74UI5FIIBwOo7y8HGazGW1tbVpfX18f+vv74XK5AAAulwvd3d3w+/3amNbWViiKgrKyskmfw2q1aqf+xzYi0r+UPoI1NjZi/fr1cDqdCAaDePvtt3Hs2DG8//77sNls2Lx5MxoaGlBQUABFUbB9+3a4XC5UVlYCANatW4eysjI89dRT2LNnD7xeL55//nm43W5YrdZ7soNENHelFEB+vx/f/e53cenSJdhsNqxYsQLvv/8+vva1rwEAXn31VRiNRtTW1iIcDqO6uhpvvPGGdn+TyYSWlhZs27YNLpcLOTk5qKurw0svvZTevdK5sXd86WI2m9P2WETpZBA6/AJJIBCAzWaDqqrz8uNYIBCAqqbvF08LCwuRnZ1994FEaTLV1yivBZuDOM9FnxVcD4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJkyG7gOkQQgAAAoGA5EqIaCJjr82x1+pkdBlAV69eBQCUlpZKroSI7iQYDMJms03ar8sAKigoAAD09/ffcecoWSAQQGlpKQYGBqAoiuxydIHHbHqEEAgGgygpKbnjOF0GkNF4Y+rKZrPxH8U0KIrC45YiHrPUTeXNASehiUgaBhARSaPLALJardi9ezesVqvsUnSFxy11PGb3lkHc7TwZEdE9ost3QEQ0PzCAiEgaBhARScMAIiJpdBlAe/fuxaJFi5CZmYmKigp0dHTILkmapqYmrFmzBnl5eSguLsaGDRvQ19eXNCYUCsHtdqOwsBC5ubmora2Fz+dLGtPf34+amhpkZ2ejuLgYO3fuRCwWm81dkebll1+GwWDAjh07tDYes1kidObQoUPCYrGIN998U/T09IgtW7aI/Px84fP5ZJcmRXV1tThw4IA4e/as6OrqEo899phwOp1ieHhYG7N161ZRWloq2traxJkzZ0RlZaV46KGHtP5YLCaWLVsmqqqqxF//+ldx5MgRUVRUJBobG2Xs0qzq6OgQixYtEitWrBDPPPOM1s5jNjt0F0Br164Vbrdbux2Px0VJSYloamqSWNXc4ff7BQDR3t4uhBBiaGhImM1m0dzcrI356KOPBADh8XiEEEIcOXJEGI1G4fV6tTH79u0TiqKIcDg8uzswi4LBoFiyZIlobW0VX/nKV7QA4jGbPbr6CBaJRNDZ2YmqqiqtzWg0oqqqCh6PR2Jlc4eqqgBuXrDb2dmJaDSadMyWLl0Kp9OpHTOPx4Ply5fDbrdrY6qrqxEIBNDT0zOL1c8ut9uNmpqapGMD8JjNJl1djHrlyhXE4/Gkv3QAsNvtOHfunKSq5o5EIoEdO3bg4YcfxrJlywAAXq8XFosF+fn5SWPtdju8Xq82ZqJjOtY3Hx06dAgffvghTp8+Pa6Px2z26CqA6M7cbjfOnj2LP//5z7JLmdMGBgbwzDPPoLW1FZmZmbLL+UzT1UewoqIimEymcWcjfD4fHA6HpKrmhvr6erS0tOCDDz7AwoULtXaHw4FIJIKhoaGk8bceM4fDMeExHeubbzo7O+H3+/Hggw8iIyMDGRkZaG9vx2uvvYaMjAzY7XYes1miqwCyWCwoLy9HW1ub1pZIJNDW1gaXyyWxMnmEEKivr8fhw4dx9OhRLF68OKm/vLwcZrM56Zj19fWhv79fO2Yulwvd3d3w+/3amNbWViiKgrKystnZkVn06KOPoru7G11dXdq2evVqbNy4Ufszj9kskT0LnqpDhw4Jq9UqDh48KHp7e8XTTz8t8vPzk85GfJZs27ZN2Gw2cezYMXHp0iVtu379ujZm69atwul0iqNHj4ozZ84Il8slXC6X1j92SnndunWiq6tLvPfee2LBggWfqVPKt54FE4LHbLboLoCEEOIXv/iFcDqdwmKxiLVr14qTJ0/KLkkaABNuBw4c0MaMjo6K73//++K+++4T2dnZ4oknnhCXLl1KepxPP/1UrF+/XmRlZYmioiLxgx/8QESj0VneG3luDyAes9nB5TiISBpdzQER0fzCACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKT5f5kLCAdpjkHiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200.0 200 [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        action = action.item()\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
